package com.meallon.tupapa;

import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class Crawler {  
	  
	/**
	 * 图片保存的目录
	 */
	public String dirName="C:/douban/";
	/**
	 * 扒取网络图片的线程数
	 */
	private  int THREAD_NUMS;
	

    public String getDirName() {
		return dirName;
	}
	public void setDirName(String dirName) {
		this.dirName = dirName;
	}

	
	public  int getTHREAD_NUMS() {
		return THREAD_NUMS;
	}
	public  void setTHREAD_NUMS(int tHREAD_NUMS) {
		THREAD_NUMS = tHREAD_NUMS;
	}

	/** 
     * ץȡ���� 
     *  
     * @return 
     * @param seeds 
     */  
	
	public Crawler(String dirName){
		this.dirName = dirName;
	}
	
	public Crawler(){
		int cores = Runtime.getRuntime().availableProcessors();
		THREAD_NUMS = cores*2;
	}
    public void crawling(String url,final Filter filter) {//开始扒取网络地址的入口
    	 final MyLinkBlockQueue LinkBlockQueue = new MyLinkBlockQueue();     
        LinkBlockQueue.addUnvisitedUrl(url);  
        ExecutorService cachedThreadPool = Executors.newCachedThreadPool(); 
        final DownLoadPic pic=new DownLoadPic();
        for(int i=0;i<THREAD_NUMS;i++){
        	 cachedThreadPool.execute(new Runnable() {
     			
     			public void run() {
     				  while (true) {  
     			 
     					  try{
     			            String visitUrl = (String) LinkBlockQueue.getUnVisitedUrl();       			          			                			         
     			          
     			            pic.downloadPic(visitUrl,dirName);  
     			          
     			           //获取该url页面的所有符合条件的链接
     			            Set<String> links = ParserHttpUrl.extracLinks(visitUrl, filter);
     			            
     			           //添加扒取的超链接进队列里
     			            for (String link : links) {
     			                
     			            	LinkBlockQueue.addUnvisitedUrl(link);  
     			            }  
     			         
     					  }catch(Exception E){
     						E.printStackTrace();
     						  continue;
     					  }
     			        }  		
     			}});

        }
       
        
      
    }  
  
}  
